# -*-coding:utf8-*-

import re
import sys
import os
from bs4 import BeautifulSoup
import requests
import shutil
import time
from lxml import etree

user_id = 1890735522
cookie = {
    "Cookie": "U_TRS1=00000092.df752548.59ce5970.397ce921; UOR=www.baidu.com,blog.sina.com.cn,; SINAGLOBAL=119.96.130.146_1506695536.949326; ULV=1517469571017:2:1:1::1506695538231; Apache=172.16.92.26_1527238367.365370; SCF=As8QL-ncx6w7J4r7OBS3nuFY_y6rW580UIY3Jq3o_DHLJpYEbuxTCb6e-2OWxYyMorDDqR_kuLnKjOFJ9DkamDI.; SUB=_2A252A76kDeRhGedG4lIW8yvJyT6IHXVVeJdsrDV_PUNbm9ANLXflkW9NURQjrCbWJaVM-ufc69uy5Xg-qFpfFxV1; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWZqsPCgbQz4ICOlL6PsOIT5NHD95Qp1h.7S0efSKzEWs4Dqc_5i--Xi-zRiKy2i--ciK.RiKy8i--Xi-i2iKLhi--RiKn7iKnfi--Xi-z4iKyFi--ci-82iKyhTCH8SFHF1F-RBntt; ALF=1558774388; sso_info=v02m6alo5qztLeMg6CzjYOYsImnlamZg7SxjoOksI2zjLWNk4iy"}
url = 'http://weibo.cn/u/%d?filter=1&page=1' % user_id
html = requests.get(url, cookies=cookie).content
print('user_id和cookie读入成功')
selector = etree.HTML(html)
pageNum = (int)(selector.xpath('//input[@name="mp"]')[0].attrib['value'])

result = ""
urllist_set = set()
word_count = 1
image_count = 1

print('ready')

sys.stdout.flush()

times = 5
one_step = pageNum / times
for step in range(times):
    if step < times - 1:
        i = step * one_step + 1
        j = (step + 1) * one_step + 1
    else:
        i = step * one_step + 1
        j = pageNum + 1
    for page in range(int(i), j):
        # 获取lxml页面
        try:
            url = 'http://weibo.cn/u/%d?filter=1&page=%d' % (user_id, page)
            lxml = requests.get(url, cookies=cookie).content
            # 文字爬取
            selector = etree.HTML(lxml)
            content = selector.xpath('//span[@class="ctt"]')
            for each in content:
                text = each.xpath('string(.)')
                if word_count >= 3:
                    text = "%d: " % (word_count - 2) + text + "\n"
                else:
                    text = text + "\n\n"
                result = result + text
                word_count += 1
            print(page, 'word ok')
            sys.stdout.flush()
            soup = BeautifulSoup(lxml, "lxml")
            urllist = soup.find_all('a', href=re.compile(r'^http://weibo.cn/mblog/oripic', re.I))
            urllist1 = soup.find_all('a', href=re.compile(r'^http://weibo.cn/mblog/picAll', re.I))
            for imgurl in urllist:
                imgurl['href'] = re.sub(r"amp;", '', imgurl['href'])
                #       print imgurl['href']
                urllist_set.add(requests.get(imgurl['href'], cookies=cookie).url)
                image_count += 1
            for imgurl_all in urllist1:
                html_content = requests.get(imgurl_all['href'], cookies=cookie).content
                soup = BeautifulSoup(html_content, "lxml")
                urllist2 = soup.find_all('a', href=re.compile(r'^/mblog/oripic', re.I))
                for imgurl in urllist2:
                    imgurl['href'] = 'http://weibo.cn' + re.sub(r"amp;", '', imgurl['href'])
                    urllist_set.add(requests.get(imgurl['href'], cookies=cookie).url)
                    image_count += 1
                image_count -= 1
            print(page, 'picurl ok')
        except:
            print(page, 'error')
        print(page, 'sleep')
        sys.stdout.flush()
        time.sleep(60)
    print('正在进行第', step + 1, u'次停顿，防止访问次数过多')
    time.sleep(300)
word_path = os.path.join(os.getcwd(), str(user_id))
image_path = os.path.join(os.getcwd(), 'weibo_image')
try:
    fo = open(os.getcwd() + "/%d" % user_id, "wb")
    fo.write(result)
    print('文字微博爬取完毕')
    link = ""
    fo2 = open(os.getcwd() + "/%s_image" % user_id, "wb")
    for eachlink in urllist_set:
        link = link + eachlink + "\n"
    fo2.write(link)
    print('图片链接爬取完毕')
except:
    print('存放数据地址有误')
sys.stdout.flush()

if not urllist_set:
    print('该用户原创微博中不存在图片')
else:
    # 下载图片,保存在当前目录的pythonimg文件夹下
    if os.path.exists(image_path) is False:
        os.mkdir(image_path)
    x = 1
    for imgurl in urllist_set:
        temp = image_path + '/%s.jpg' % x
        print('正在下载第%s张图片' % x)
        try:
            # urllib.urlretrieve(urllib2.urlopen(imgurl).geturl(),temp)
            r = requests.get(imgurl, stream=True)
            if r.status_code == 200:
                with open(temp, 'wb') as f:
                    r.raw.decode_content = True
                    shutil.copyfileobj(r.raw, f)
        except:
            print("该图片下载失败:%s" % imgurl)

        x += 1
print('原创微博爬取完毕，共%d条，保存路径%s' % (word_count - 3, word_path))
print('微博图片爬取完毕，共%d张，保存路径%s' % (image_count - 1, image_path))
